﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using log4net;
using HtmlAgilityPack;
using DWH.Utils;
using DWH.Data;
using System.IO;
using System.Threading;

namespace DWH.PP.LoopUrls
{
    public class PPLoopUrlsBoardPullTask : PPBoardPullTaskBase
    {

        protected string loopPageListUrlPattern; //"http://70.85.48.252/bt/thread.php?fid=14&search=&page={0}";
        protected string boardUrlRootPrefix;
        protected int loopPageListNumberStart = 1;
        protected int loopPageListNumberEnd = 100;
        protected bool pageHistoryDetectAndRestart = false;
        protected int pageHistoryDetectNumber = 1000;
        protected string pageTargetElement = "//a[@href]";
        protected string pageTargetAttribute = "href";

        protected string attachmentStoreFolder;
        protected string currentProcessingPageListUrl;

        private int conflictPage = 0;                
        private Dictionary<int, string> visitedPageContentUrls;        

        private ILog logger;       

        public override void Initialize()
        {
            base.Initialize();

            logger = LogManager.GetLogger(typeof(PPLoopUrlsBoardPullTask));

            loopPageListUrlPattern = ExtendData[DWHConstants.KEY_LOOP_PAGELIST_URL_PATTERN];
            loopPageListNumberStart = Int32.Parse(ExtendData[DWHConstants.KEY_LOOP_PAGELIST_NUMBER_START]);
            loopPageListNumberEnd = Int32.Parse(ExtendData[DWHConstants.KEY_LOOP_PAGELIST_NUMBER_END]);

            if (ExtendData.ContainsKey(DWHConstants.KEY_PAGE_HISTORY_DETECT_RESTART))
            {
                pageHistoryDetectAndRestart = Boolean.Parse(ExtendData[DWHConstants.KEY_PAGE_HISTORY_DETECT_RESTART]);
            }

            if (ExtendData.ContainsKey(DWHConstants.KEY_PAGE_HISTORY_DETECT_NUMBER))
            {
                pageHistoryDetectNumber = Int32.Parse(ExtendData[DWHConstants.KEY_PAGE_HISTORY_DETECT_NUMBER]);
            }
            else
            {
                pageHistoryDetectNumber = DWHConstants.V_PAGE_HISTORY_RESTART_DEFAULT;
            }

            if (ExtendData.ContainsKey(DWHConstants.KEY_PAGE_TARGET_ELEMENT))
            {
                pageTargetElement = ExtendData[DWHConstants.KEY_PAGE_TARGET_ELEMENT];
            }

            if (ExtendData.ContainsKey(DWHConstants.KEY_PAGE_TARGET_ATTRIBUTE))
            {
                pageTargetAttribute = ExtendData[DWHConstants.KEY_PAGE_TARGET_ATTRIBUTE];
            }

            if (ExtendData.ContainsKey(DWHConstants.KEY_WEB_ATTACHMENT_STORE_FOLDER))
            {
                attachmentStoreFolder = ExtendData[DWHConstants.KEY_WEB_ATTACHMENT_STORE_FOLDER];

                //if (!Directory.Exists(attachmentStoreFolder))
                //{
                //    Directory.CreateDirectory(attachmentStoreFolder);
                //} 
                //move download attachment to another task, skip the foder creation
            }

            boardUrlRootPrefix = loopPageListUrlPattern.Substring(0, loopPageListUrlPattern.LastIndexOf('/') + 1);
            
            visitedPageContentUrls = new Dictionary<int, string>();
        }

        public override void Run()
        {
            //while (true)
            //{
            logger.InfoFormat("task {0} started", TaskName);            

            for (int i = loopPageListNumberStart; i <= loopPageListNumberEnd; i++)
            {
                try
                {
                    currentProcessingPageListUrl = String.Format(loopPageListUrlPattern, i);
                    ProcessPageList(currentProcessingPageListUrl);

                    //Thread.Sleep(new Random().Next(5) * 1000); //sleep for a while to simulate user visit   
                    DWHHelper.SimulateUserClickInterval(5);
                }
                catch (Exception ex)
                {
                    logger.ErrorFormat("Error occurs when processing page list:{0} , {1}", currentProcessingPageListUrl, ex.ToString());
                }
            }

            logger.InfoFormat("task {0} finished for this run, ", TaskName);

            //    visitedPageListUrls.Clear();
            //    conflictPage = 0;
            //}
        }

        public override void Reset()
        {
            logger.InfoFormat("reset task {0}", TaskName);
        }

        public override void ProcessPageList(string currentPageListUrl)
        {
            logger.InfoFormat("task {0}, processing page list:{1}", TaskName, currentPageListUrl);            

            HtmlWeb web = new HtmlWeb();
            web.OverrideEncoding = Encoding.GetEncoding(pageEncoding);

            HtmlDocument doc = null;
            for (int i = 0; i < 3; i++)
            {
                try
                {
                    doc = web.Load(currentPageListUrl);
                    break;
                }
                catch (Exception ex)
                {
                    logger.ErrorFormat("Error occurs when loading page list:{0} , {1}", currentPageListUrl, ex.ToString());
                }
            }

            if (doc == null)
            {
                logger.ErrorFormat("failed to load page list with three try:{0}", currentPageListUrl);
                return;
            }

            List<string> tempPageContentUrls;
            List<string> tempPageListUrls;
            tempPageContentUrls = new List<string>();
            tempPageListUrls = new List<string>();

            HtmlNodeCollection canLinkColl = doc.DocumentNode.SelectNodes(pageTargetElement);

            List<string> canLinkUrls = new List<string>();
            foreach (HtmlNode link in canLinkColl)
            {
                HtmlAttribute att = link.Attributes[pageTargetAttribute];
                if (att != null && !String.IsNullOrEmpty(att.Value))
                {
                    canLinkUrls.Add(att.Value);
                }
            }

            foreach (string rlinkUrl in canLinkUrls)
            {
                //string linkUrl = link.Attributes["href"].Value;

                if (DWHHelper.PatternMatch(rlinkUrl, contentPageUrlPattern))
                {
                    string flinkUrl = DWHHelper.FormatUrl(boardUrlRootPrefix, rlinkUrl);
                    //add page content
                    if (visitedPageContentUrls.ContainsKey(flinkUrl.GetHashCode()))
                    {
                        conflictPage++;
                        logger.InfoFormat("task {0} ,this page has been processed in this run: {1}", TaskName, flinkUrl);
                    }
                    else
                    {
                        if (!tempPageContentUrls.Contains(flinkUrl))
                        {
                            tempPageContentUrls.Add(flinkUrl);
                        }
                    }
                }
            }

            //process page content first            
            foreach (string pageContentUrl in tempPageContentUrls)
            {
                if (DWHDBHelper.IsPageVisited(pageContentUrl.GetHashCode()))
                {
                    conflictPage++;
                    logger.InfoFormat("task {0} ,this page has been processed in previous run: {1}", TaskName, pageContentUrl);
                }
                else
                {
                    //if (!DWHDBHelper.IsPageVisited(pageContentUrl.GetHashCode()) 
                    //    && !visitedPageContentUrls.ContainsKey(pageContentUrl.GetHashCode()))

                    logger.InfoFormat("task {0} ,processing page: {1}", TaskName, pageContentUrl);

                    try
                    {
                        visitedPageContentUrls.Add(pageContentUrl.GetHashCode(), pageContentUrl);
                        ProcessPageContent(pageContentUrl);
                        //Thread.Sleep(new Random().Next(3) * 1000); //sleep for a while to simulate user visit  
                        DWHHelper.SimulateUserClickInterval(3);
                    }
                    catch (Exception ex)
                    {
                        logger.ErrorFormat("Error occurs when processing pageContentUrl:{0} , {1}", pageContentUrl, ex.ToString());
                    }
                }
            }

            DWHDBHelper.AddOrUpdatePageList(TaskName, currentPageListUrl.GetHashCode(), currentPageListUrl, DateTime.Now);
            DWHDBHelper.UpdatePPTask(TaskName, PPTaskResumeData);

            //if done, restart;
            if (pageHistoryDetectAndRestart && (conflictPage > pageHistoryDetectNumber))
            {
                logger.InfoFormat("task {0} has come to conflict page for restart, restart", TaskName);                
                conflictPage = 0;
                Run();
            }                    
        }
    }
}
